{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "import itertools\n",
    "import collections\n",
    "import re\n",
    "import random\n",
    "import sentencepiece as spm\n",
    "from unidecode import unidecode\n",
    "from sklearn.utils import shuffle\n",
    "from tqdm import tqdm\n",
    "from prepro_utils import preprocess_text, encode_ids, encode_pieces\n",
    "from malaya.text.function import transformer_textcleaning as cleaning\n",
    "from tensorflow.python.estimator.run_config import RunConfig\n",
    "import bert_utils as squad_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import logging\n",
    "\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
    "tf.get_logger().setLevel(logging.ERROR)\n",
    "tf.autograph.set_verbosity(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('sp10m.cased.bert.model')\n",
    "\n",
    "with open('sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v, sp_model):\n",
    "        self.vocab = v\n",
    "        self.sp_model = sp_model\n",
    "\n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(\n",
    "            self.sp_model, string, return_unicode = False, sample = False\n",
    "        )\n",
    "\n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [self.sp_model.PieceToId(piece) for piece in tokens]\n",
    "\n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [self.sp_model.IdToPiece(i) for i in ids]\n",
    "\n",
    "\n",
    "tokenizer = Tokenizer(v, sp_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('bert-squad-test.pkl', 'rb') as fopen:\n",
    "    test_features, test_examples = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'qas_id': '56ddde6b9a695914005b9628',\n",
       " 'question_text': 'Di negara manakah Normandy berada?',\n",
       " 'paragraph_text': 'Orang Norman (Norman: Nourmands; Perancis: Normands; Latin: Normanni) ialah orang-orang yang pada abad ke-10 dan ke-11 memberikan nama mereka kepada Normandy, sebuah wilayah di Perancis. Mereka diturunkan daripada Norse (\"Norman\" berasal daripada penyerang \"Norseman\") dan lanun dari Denmark, Iceland dan Norway yang, di bawah pimpinan mereka Rollo, bersetuju untuk bersumpah fealty kepada Raja Charles III dari Francia Barat. Melalui generasi asimilasi dan percampuran dengan penduduk asli Frankish dan Roman-Gaulish, keturunan mereka akan beransur-ansur bergabung dengan budaya Carolingian yang berpusat di Francia Barat. Identiti budaya dan etnik yang berbeza dari orang Norman muncul pada mulanya pada separuh pertama abad ke-10, dan ia terus berkembang pada abad-abad yang berjaya.',\n",
       " 'orig_answer_text': None,\n",
       " 'start_position': None,\n",
       " 'end_position': None,\n",
       " 'is_impossible': False}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_examples[0].__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_seq_length = 384\n",
    "doc_stride = 128\n",
    "max_query_length = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(\n",
    "    'tiny-bert-v1/config.json'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.contrib import layers as contrib_layers\n",
    "\n",
    "class Model:\n",
    "    def __init__(self, is_training = True):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.segment_ids = tf.placeholder(tf.int32, [None, None])\n",
    "        self.input_masks = tf.placeholder(tf.int32, [None, None])\n",
    "        self.p_mask = tf.placeholder(tf.int32, [None, None])\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=is_training,\n",
    "            input_ids=self.X,\n",
    "            input_mask=self.input_masks,\n",
    "            token_type_ids=self.segment_ids,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        final_hidden = model.get_sequence_output()\n",
    "        self.output = final_hidden\n",
    "        vectorize = tf.identity(final_hidden, name = 'logits_vectorize')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 2e-5\n",
    "start_n_top = 5\n",
    "end_n_top = 5\n",
    "is_training = False\n",
    "\n",
    "tf.reset_default_graph()\n",
    "model = Model(is_training = is_training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = model.output\n",
    "bsz = tf.shape(output)[0]\n",
    "return_dict = {}\n",
    "output = tf.transpose(output, [1, 0, 2])\n",
    "\n",
    "# invalid position mask such as query and special symbols (PAD, SEP, CLS)\n",
    "p_mask = tf.cast(model.p_mask, dtype = tf.float32)\n",
    "\n",
    "# logit of the start position\n",
    "with tf.variable_scope('start_logits'):\n",
    "    start_logits = tf.layers.dense(\n",
    "        output,\n",
    "        1,\n",
    "        kernel_initializer = modeling.create_initializer(\n",
    "            bert_config.initializer_range\n",
    "        ),\n",
    "    )\n",
    "    start_logits = tf.transpose(tf.squeeze(start_logits, -1), [1, 0])\n",
    "    start_logits_masked = start_logits * (1 - p_mask) - 1e30 * p_mask\n",
    "    start_log_probs = tf.nn.log_softmax(start_logits_masked, -1)\n",
    "\n",
    "# logit of the end position\n",
    "with tf.variable_scope('end_logits'):\n",
    "    if is_training:\n",
    "        # during training, compute the end logits based on the\n",
    "        # ground truth of the start position\n",
    "        start_positions = tf.reshape(model.start_positions, [-1])\n",
    "        start_index = tf.one_hot(\n",
    "            start_positions,\n",
    "            depth = max_seq_length,\n",
    "            axis = -1,\n",
    "            dtype = tf.float32,\n",
    "        )\n",
    "        start_features = tf.einsum('lbh,bl->bh', output, start_index)\n",
    "        start_features = tf.tile(\n",
    "            start_features[None], [max_seq_length, 1, 1]\n",
    "        )\n",
    "        end_logits = tf.layers.dense(\n",
    "            tf.concat([output, start_features], axis = -1),\n",
    "            bert_config.hidden_size,\n",
    "            kernel_initializer = modeling.create_initializer(\n",
    "                bert_config.initializer_range\n",
    "            ),\n",
    "            activation = tf.tanh,\n",
    "            name = 'dense_0',\n",
    "        )\n",
    "        end_logits = contrib_layers.layer_norm(\n",
    "            end_logits, begin_norm_axis = -1\n",
    "        )\n",
    "\n",
    "        end_logits = tf.layers.dense(\n",
    "            end_logits,\n",
    "            1,\n",
    "            kernel_initializer = modeling.create_initializer(\n",
    "                bert_config.initializer_range\n",
    "            ),\n",
    "            name = 'dense_1',\n",
    "        )\n",
    "        end_logits = tf.transpose(tf.squeeze(end_logits, -1), [1, 0])\n",
    "        end_logits_masked = end_logits * (1 - p_mask) - 1e30 * p_mask\n",
    "        end_log_probs = tf.nn.log_softmax(end_logits_masked, -1)\n",
    "    else:\n",
    "        # during inference, compute the end logits based on beam search\n",
    "\n",
    "        start_top_log_probs, start_top_index = tf.nn.top_k(\n",
    "            start_log_probs, k = start_n_top\n",
    "        )\n",
    "        start_index = tf.one_hot(\n",
    "            start_top_index,\n",
    "            depth = max_seq_length,\n",
    "            axis = -1,\n",
    "            dtype = tf.float32,\n",
    "        )\n",
    "        start_features = tf.einsum('lbh,bkl->bkh', output, start_index)\n",
    "        end_input = tf.tile(output[:, :, None], [1, 1, start_n_top, 1])\n",
    "        start_features = tf.tile(\n",
    "            start_features[None], [max_seq_length, 1, 1, 1]\n",
    "        )\n",
    "        end_input = tf.concat([end_input, start_features], axis = -1)\n",
    "        end_logits = tf.layers.dense(\n",
    "            end_input,\n",
    "            bert_config.hidden_size,\n",
    "            kernel_initializer = modeling.create_initializer(\n",
    "                bert_config.initializer_range\n",
    "            ),\n",
    "            activation = tf.tanh,\n",
    "            name = 'dense_0',\n",
    "        )\n",
    "        end_logits = contrib_layers.layer_norm(\n",
    "            end_logits, begin_norm_axis = -1\n",
    "        )\n",
    "        end_logits = tf.layers.dense(\n",
    "            end_logits,\n",
    "            1,\n",
    "            kernel_initializer = modeling.create_initializer(\n",
    "                bert_config.initializer_range\n",
    "            ),\n",
    "            name = 'dense_1',\n",
    "        )\n",
    "        end_logits = tf.reshape(\n",
    "            end_logits, [max_seq_length, -1, start_n_top]\n",
    "        )\n",
    "        end_logits = tf.transpose(end_logits, [1, 2, 0])\n",
    "        end_logits_masked = (\n",
    "            end_logits * (1 - p_mask[:, None]) - 1e30 * p_mask[:, None]\n",
    "        )\n",
    "        end_log_probs = tf.nn.log_softmax(end_logits_masked, -1)\n",
    "        end_top_log_probs, end_top_index = tf.nn.top_k(\n",
    "            end_log_probs, k = end_n_top\n",
    "        )\n",
    "        end_top_log_probs = tf.reshape(\n",
    "            end_top_log_probs, [-1, start_n_top * end_n_top]\n",
    "        )\n",
    "        end_top_index = tf.reshape(\n",
    "            end_top_index, [-1, start_n_top * end_n_top]\n",
    "        )\n",
    "        \n",
    "if is_training:\n",
    "    return_dict['start_log_probs'] = start_log_probs\n",
    "    return_dict['end_log_probs'] = end_log_probs\n",
    "else:\n",
    "    return_dict['start_top_log_probs'] = start_top_log_probs\n",
    "    return_dict['start_top_index'] = start_top_index\n",
    "    return_dict['end_top_log_probs'] = end_top_log_probs\n",
    "    return_dict['end_top_index'] = end_top_index\n",
    "\n",
    "# an additional layer to predict answerability\n",
    "with tf.variable_scope('answer_class'):\n",
    "    # get the representation of CLS\n",
    "    cls_index = tf.one_hot(\n",
    "        tf.zeros([bsz], dtype = tf.int32),\n",
    "        max_seq_length,\n",
    "        axis = -1,\n",
    "        dtype = tf.float32,\n",
    "    )\n",
    "    cls_feature = tf.einsum('lbh,bl->bh', output, cls_index)\n",
    "\n",
    "    # get the representation of START\n",
    "    start_p = tf.nn.softmax(\n",
    "        start_logits_masked, axis = -1, name = 'softmax_start'\n",
    "    )\n",
    "    start_feature = tf.einsum('lbh,bl->bh', output, start_p)\n",
    "\n",
    "    # note(zhiliny): no dependency on end_feature so that we can obtain\n",
    "    # one single `cls_logits` for each sample\n",
    "    ans_feature = tf.concat([start_feature, cls_feature], -1)\n",
    "    ans_feature = tf.layers.dense(\n",
    "        ans_feature,\n",
    "        bert_config.hidden_size,\n",
    "        activation = tf.tanh,\n",
    "        kernel_initializer = modeling.create_initializer(\n",
    "            bert_config.initializer_range\n",
    "        ),\n",
    "        name = 'dense_0',\n",
    "    )\n",
    "    ans_feature = tf.layers.dropout(\n",
    "        ans_feature, bert_config.hidden_dropout_prob, training = is_training\n",
    "    )\n",
    "    cls_logits = tf.layers.dense(\n",
    "        ans_feature,\n",
    "        1,\n",
    "        kernel_initializer = modeling.create_initializer(\n",
    "            bert_config.initializer_range\n",
    "        ),\n",
    "        name = 'dense_1',\n",
    "        use_bias = False,\n",
    "    )\n",
    "    cls_logits = tf.squeeze(cls_logits, -1)\n",
    "    \n",
    "return_dict['cls_logits'] = cls_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(var_list = tf.trainable_variables())\n",
    "saver.restore(sess, 'tiny-bert-squad/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_top_log_probs = tf.identity(start_top_log_probs, name = 'start_top_log_probs')\n",
    "start_top_index = tf.identity(start_top_index, name = 'start_top_index')\n",
    "end_top_log_probs = tf.identity(end_top_log_probs, name = 'end_top_log_probs')\n",
    "end_top_index = tf.identity(end_top_index, name = 'end_top_index')\n",
    "cls_logits = tf.identity(cls_logits, name = 'cls_logits')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 0\n",
    "batch_size = 2\n",
    "batch = test_features[i: i + batch_size]\n",
    "batch_ids = [b.input_ids for b in batch]\n",
    "batch_masks = [b.input_mask for b in batch]\n",
    "batch_segment = [b.segment_ids for b in batch]\n",
    "batch_start = [b.start_position for b in batch]\n",
    "batch_end = [b.end_position for b in batch]\n",
    "is_impossible = [b.is_impossible for b in batch]\n",
    "p_mask = [b.p_mask for b in batch]\n",
    "o = sess.run(\n",
    "    [start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits],\n",
    "    feed_dict = {\n",
    "        model.X: batch_ids,\n",
    "        model.segment_ids: batch_segment,\n",
    "        model.input_masks: batch_masks,\n",
    "        model.p_mask: p_mask\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'output-tiny-bert-squad/model.ckpt'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'output-tiny-bert-squad/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'Placeholder_3',\n",
       " 'bert/embeddings/word_embeddings',\n",
       " 'bert/embeddings/token_type_embeddings',\n",
       " 'bert/embeddings/position_embeddings',\n",
       " 'bert/embeddings/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/query/bias',\n",
       " 'bert/encoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/key/bias',\n",
       " 'bert/encoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/value/bias',\n",
       " 'bert/encoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_0/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/output/dense/bias',\n",
       " 'bert/encoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/query/bias',\n",
       " 'bert/encoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/key/bias',\n",
       " 'bert/encoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/value/bias',\n",
       " 'bert/encoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_1/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/output/dense/bias',\n",
       " 'bert/encoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/query/bias',\n",
       " 'bert/encoder/layer_2/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/key/bias',\n",
       " 'bert/encoder/layer_2/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/value/bias',\n",
       " 'bert/encoder/layer_2/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_2/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_2/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_2/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/output/dense/bias',\n",
       " 'bert/encoder/layer_2/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/query/bias',\n",
       " 'bert/encoder/layer_3/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/key/bias',\n",
       " 'bert/encoder/layer_3/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/value/bias',\n",
       " 'bert/encoder/layer_3/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_3/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_3/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_3/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/output/dense/bias',\n",
       " 'bert/encoder/layer_3/output/LayerNorm/gamma',\n",
       " 'bert/pooler/dense/kernel',\n",
       " 'bert/pooler/dense/bias',\n",
       " 'logits_vectorize',\n",
       " 'start_logits/dense/kernel/Initializer/truncated_normal/shape',\n",
       " 'start_logits/dense/kernel/Initializer/truncated_normal/mean',\n",
       " 'start_logits/dense/kernel/Initializer/truncated_normal/stddev',\n",
       " 'start_logits/dense/kernel/Initializer/truncated_normal/TruncatedNormal',\n",
       " 'start_logits/dense/kernel/Initializer/truncated_normal/mul',\n",
       " 'start_logits/dense/kernel/Initializer/truncated_normal',\n",
       " 'start_logits/dense/kernel',\n",
       " 'start_logits/dense/kernel/Assign',\n",
       " 'start_logits/dense/kernel/read',\n",
       " 'start_logits/dense/bias/Initializer/zeros',\n",
       " 'start_logits/dense/bias',\n",
       " 'start_logits/dense/bias/Assign',\n",
       " 'start_logits/dense/bias/read',\n",
       " 'start_logits/dense/Tensordot/axes',\n",
       " 'start_logits/dense/Tensordot/free',\n",
       " 'start_logits/dense/Tensordot/Shape',\n",
       " 'start_logits/dense/Tensordot/GatherV2/axis',\n",
       " 'start_logits/dense/Tensordot/GatherV2',\n",
       " 'start_logits/dense/Tensordot/GatherV2_1/axis',\n",
       " 'start_logits/dense/Tensordot/GatherV2_1',\n",
       " 'start_logits/dense/Tensordot/Const',\n",
       " 'start_logits/dense/Tensordot/Prod',\n",
       " 'start_logits/dense/Tensordot/Const_1',\n",
       " 'start_logits/dense/Tensordot/Prod_1',\n",
       " 'start_logits/dense/Tensordot/concat/axis',\n",
       " 'start_logits/dense/Tensordot/concat',\n",
       " 'start_logits/dense/Tensordot/stack',\n",
       " 'start_logits/dense/Tensordot/transpose',\n",
       " 'start_logits/dense/Tensordot/Reshape',\n",
       " 'start_logits/dense/Tensordot/transpose_1/perm',\n",
       " 'start_logits/dense/Tensordot/transpose_1',\n",
       " 'start_logits/dense/Tensordot/Reshape_1/shape',\n",
       " 'start_logits/dense/Tensordot/Reshape_1',\n",
       " 'start_logits/dense/Tensordot/MatMul',\n",
       " 'start_logits/dense/Tensordot/Const_2',\n",
       " 'start_logits/dense/Tensordot/concat_1/axis',\n",
       " 'start_logits/dense/Tensordot/concat_1',\n",
       " 'start_logits/dense/Tensordot',\n",
       " 'start_logits/dense/BiasAdd',\n",
       " 'start_logits/Squeeze',\n",
       " 'start_logits/transpose/perm',\n",
       " 'start_logits/transpose',\n",
       " 'start_logits/sub/x',\n",
       " 'start_logits/sub',\n",
       " 'start_logits/mul',\n",
       " 'start_logits/mul_1/x',\n",
       " 'start_logits/mul_1',\n",
       " 'start_logits/sub_1',\n",
       " 'start_logits/LogSoftmax',\n",
       " 'end_logits/TopKV2/k',\n",
       " 'end_logits/TopKV2',\n",
       " 'end_logits/one_hot/on_value',\n",
       " 'end_logits/one_hot/off_value',\n",
       " 'end_logits/one_hot/depth',\n",
       " 'end_logits/one_hot',\n",
       " 'end_logits/einsum/transpose/perm',\n",
       " 'end_logits/einsum/transpose',\n",
       " 'end_logits/einsum/transpose_1/perm',\n",
       " 'end_logits/einsum/transpose_1',\n",
       " 'end_logits/einsum/Shape',\n",
       " 'end_logits/einsum/strided_slice/stack',\n",
       " 'end_logits/einsum/strided_slice/stack_1',\n",
       " 'end_logits/einsum/strided_slice/stack_2',\n",
       " 'end_logits/einsum/strided_slice',\n",
       " 'end_logits/einsum/strided_slice_1/stack',\n",
       " 'end_logits/einsum/strided_slice_1/stack_1',\n",
       " 'end_logits/einsum/strided_slice_1/stack_2',\n",
       " 'end_logits/einsum/strided_slice_1',\n",
       " 'end_logits/einsum/mul/x',\n",
       " 'end_logits/einsum/mul',\n",
       " 'end_logits/einsum/Reshape/shape/1',\n",
       " 'end_logits/einsum/Reshape/shape',\n",
       " 'end_logits/einsum/Reshape',\n",
       " 'end_logits/einsum/Shape_1',\n",
       " 'end_logits/einsum/strided_slice_2/stack',\n",
       " 'end_logits/einsum/strided_slice_2/stack_1',\n",
       " 'end_logits/einsum/strided_slice_2/stack_2',\n",
       " 'end_logits/einsum/strided_slice_2',\n",
       " 'end_logits/einsum/Reshape_1/shape/2',\n",
       " 'end_logits/einsum/Reshape_1/shape',\n",
       " 'end_logits/einsum/Reshape_1',\n",
       " 'end_logits/einsum/MatMul',\n",
       " 'end_logits/einsum/Reshape_2/shape/1',\n",
       " 'end_logits/einsum/Reshape_2/shape/2',\n",
       " 'end_logits/einsum/Reshape_2/shape',\n",
       " 'end_logits/einsum/Reshape_2',\n",
       " 'end_logits/einsum/transpose_2/perm',\n",
       " 'end_logits/einsum/transpose_2',\n",
       " 'end_logits/strided_slice/stack',\n",
       " 'end_logits/strided_slice/stack_1',\n",
       " 'end_logits/strided_slice/stack_2',\n",
       " 'end_logits/strided_slice',\n",
       " 'end_logits/Tile/multiples',\n",
       " 'end_logits/Tile',\n",
       " 'end_logits/strided_slice_1/stack',\n",
       " 'end_logits/strided_slice_1/stack_1',\n",
       " 'end_logits/strided_slice_1/stack_2',\n",
       " 'end_logits/strided_slice_1',\n",
       " 'end_logits/Tile_1/multiples',\n",
       " 'end_logits/Tile_1',\n",
       " 'end_logits/concat/axis',\n",
       " 'end_logits/concat',\n",
       " 'end_logits/dense_0/kernel/Initializer/truncated_normal/shape',\n",
       " 'end_logits/dense_0/kernel/Initializer/truncated_normal/mean',\n",
       " 'end_logits/dense_0/kernel/Initializer/truncated_normal/stddev',\n",
       " 'end_logits/dense_0/kernel/Initializer/truncated_normal/TruncatedNormal',\n",
       " 'end_logits/dense_0/kernel/Initializer/truncated_normal/mul',\n",
       " 'end_logits/dense_0/kernel/Initializer/truncated_normal',\n",
       " 'end_logits/dense_0/kernel',\n",
       " 'end_logits/dense_0/kernel/Assign',\n",
       " 'end_logits/dense_0/kernel/read',\n",
       " 'end_logits/dense_0/bias/Initializer/zeros',\n",
       " 'end_logits/dense_0/bias',\n",
       " 'end_logits/dense_0/bias/Assign',\n",
       " 'end_logits/dense_0/bias/read',\n",
       " 'end_logits/dense_0/Tensordot/axes',\n",
       " 'end_logits/dense_0/Tensordot/free',\n",
       " 'end_logits/dense_0/Tensordot/Shape',\n",
       " 'end_logits/dense_0/Tensordot/GatherV2/axis',\n",
       " 'end_logits/dense_0/Tensordot/GatherV2',\n",
       " 'end_logits/dense_0/Tensordot/GatherV2_1/axis',\n",
       " 'end_logits/dense_0/Tensordot/GatherV2_1',\n",
       " 'end_logits/dense_0/Tensordot/Const',\n",
       " 'end_logits/dense_0/Tensordot/Prod',\n",
       " 'end_logits/dense_0/Tensordot/Const_1',\n",
       " 'end_logits/dense_0/Tensordot/Prod_1',\n",
       " 'end_logits/dense_0/Tensordot/concat/axis',\n",
       " 'end_logits/dense_0/Tensordot/concat',\n",
       " 'end_logits/dense_0/Tensordot/stack',\n",
       " 'end_logits/dense_0/Tensordot/transpose',\n",
       " 'end_logits/dense_0/Tensordot/Reshape',\n",
       " 'end_logits/dense_0/Tensordot/transpose_1/perm',\n",
       " 'end_logits/dense_0/Tensordot/transpose_1',\n",
       " 'end_logits/dense_0/Tensordot/Reshape_1/shape',\n",
       " 'end_logits/dense_0/Tensordot/Reshape_1',\n",
       " 'end_logits/dense_0/Tensordot/MatMul',\n",
       " 'end_logits/dense_0/Tensordot/Const_2',\n",
       " 'end_logits/dense_0/Tensordot/concat_1/axis',\n",
       " 'end_logits/dense_0/Tensordot/concat_1',\n",
       " 'end_logits/dense_0/Tensordot',\n",
       " 'end_logits/dense_0/BiasAdd',\n",
       " 'end_logits/dense_0/Tanh',\n",
       " 'end_logits/LayerNorm/gamma/Initializer/ones',\n",
       " 'end_logits/LayerNorm/gamma',\n",
       " 'end_logits/LayerNorm/gamma/Assign',\n",
       " 'end_logits/LayerNorm/gamma/read',\n",
       " 'end_logits/LayerNorm/moments/mean/reduction_indices',\n",
       " 'end_logits/LayerNorm/moments/mean',\n",
       " 'end_logits/LayerNorm/moments/StopGradient',\n",
       " 'end_logits/LayerNorm/moments/SquaredDifference',\n",
       " 'end_logits/LayerNorm/moments/variance/reduction_indices',\n",
       " 'end_logits/LayerNorm/moments/variance',\n",
       " 'end_logits/LayerNorm/batchnorm/add/y',\n",
       " 'end_logits/LayerNorm/batchnorm/add',\n",
       " 'end_logits/LayerNorm/batchnorm/Rsqrt',\n",
       " 'end_logits/LayerNorm/batchnorm/mul',\n",
       " 'end_logits/LayerNorm/batchnorm/mul_1',\n",
       " 'end_logits/LayerNorm/batchnorm/mul_2',\n",
       " 'end_logits/LayerNorm/batchnorm/sub',\n",
       " 'end_logits/LayerNorm/batchnorm/add_1',\n",
       " 'end_logits/dense_1/kernel/Initializer/truncated_normal/shape',\n",
       " 'end_logits/dense_1/kernel/Initializer/truncated_normal/mean',\n",
       " 'end_logits/dense_1/kernel/Initializer/truncated_normal/stddev',\n",
       " 'end_logits/dense_1/kernel/Initializer/truncated_normal/TruncatedNormal',\n",
       " 'end_logits/dense_1/kernel/Initializer/truncated_normal/mul',\n",
       " 'end_logits/dense_1/kernel/Initializer/truncated_normal',\n",
       " 'end_logits/dense_1/kernel',\n",
       " 'end_logits/dense_1/kernel/Assign',\n",
       " 'end_logits/dense_1/kernel/read',\n",
       " 'end_logits/dense_1/bias/Initializer/zeros',\n",
       " 'end_logits/dense_1/bias',\n",
       " 'end_logits/dense_1/bias/Assign',\n",
       " 'end_logits/dense_1/bias/read',\n",
       " 'end_logits/dense_1/Tensordot/axes',\n",
       " 'end_logits/dense_1/Tensordot/free',\n",
       " 'end_logits/dense_1/Tensordot/Shape',\n",
       " 'end_logits/dense_1/Tensordot/GatherV2/axis',\n",
       " 'end_logits/dense_1/Tensordot/GatherV2',\n",
       " 'end_logits/dense_1/Tensordot/GatherV2_1/axis',\n",
       " 'end_logits/dense_1/Tensordot/GatherV2_1',\n",
       " 'end_logits/dense_1/Tensordot/Const',\n",
       " 'end_logits/dense_1/Tensordot/Prod',\n",
       " 'end_logits/dense_1/Tensordot/Const_1',\n",
       " 'end_logits/dense_1/Tensordot/Prod_1',\n",
       " 'end_logits/dense_1/Tensordot/concat/axis',\n",
       " 'end_logits/dense_1/Tensordot/concat',\n",
       " 'end_logits/dense_1/Tensordot/stack',\n",
       " 'end_logits/dense_1/Tensordot/transpose',\n",
       " 'end_logits/dense_1/Tensordot/Reshape',\n",
       " 'end_logits/dense_1/Tensordot/transpose_1/perm',\n",
       " 'end_logits/dense_1/Tensordot/transpose_1',\n",
       " 'end_logits/dense_1/Tensordot/Reshape_1/shape',\n",
       " 'end_logits/dense_1/Tensordot/Reshape_1',\n",
       " 'end_logits/dense_1/Tensordot/MatMul',\n",
       " 'end_logits/dense_1/Tensordot/Const_2',\n",
       " 'end_logits/dense_1/Tensordot/concat_1/axis',\n",
       " 'end_logits/dense_1/Tensordot/concat_1',\n",
       " 'end_logits/dense_1/Tensordot',\n",
       " 'end_logits/dense_1/BiasAdd',\n",
       " 'end_logits/Reshape/shape',\n",
       " 'end_logits/Reshape',\n",
       " 'end_logits/transpose/perm',\n",
       " 'end_logits/transpose',\n",
       " 'end_logits/strided_slice_2/stack',\n",
       " 'end_logits/strided_slice_2/stack_1',\n",
       " 'end_logits/strided_slice_2/stack_2',\n",
       " 'end_logits/strided_slice_2',\n",
       " 'end_logits/sub/x',\n",
       " 'end_logits/sub',\n",
       " 'end_logits/mul',\n",
       " 'end_logits/strided_slice_3/stack',\n",
       " 'end_logits/strided_slice_3/stack_1',\n",
       " 'end_logits/strided_slice_3/stack_2',\n",
       " 'end_logits/strided_slice_3',\n",
       " 'end_logits/mul_1/x',\n",
       " 'end_logits/mul_1',\n",
       " 'end_logits/sub_1',\n",
       " 'end_logits/LogSoftmax',\n",
       " 'end_logits/TopKV2_1/k',\n",
       " 'end_logits/TopKV2_1',\n",
       " 'end_logits/Reshape_1/shape',\n",
       " 'end_logits/Reshape_1',\n",
       " 'end_logits/Reshape_2/shape',\n",
       " 'end_logits/Reshape_2',\n",
       " 'answer_class/dense_0/kernel',\n",
       " 'answer_class/dense_0/bias',\n",
       " 'answer_class/dense_1/kernel',\n",
       " 'start_top_log_probs',\n",
       " 'start_top_index',\n",
       " 'end_top_log_probs',\n",
       " 'end_top_index',\n",
       " 'cls_logits']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'start_' in n.name\n",
    "        or 'end_' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "932 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('output-tiny-bert-squad', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "        \n",
    "    for node in graph_def.node:\n",
    "        if node.op == 'RefSwitch':\n",
    "            node.op = 'Switch'\n",
    "            for index in xrange(len(node.input)):\n",
    "                if 'moving_' in node.input[index]:\n",
    "                    node.input[index] = node.input[index] + '/read'\n",
    "        elif node.op == 'AssignSub':\n",
    "            node.op = 'Sub'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "        elif node.op == 'AssignAdd':\n",
    "            node.op = 'Add'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "        elif node.op == 'Assign':\n",
    "            node.op = 'Identity'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "            if 'validate_shape' in node.attr:\n",
    "                del node.attr['validate_shape']\n",
    "            if len(node.input) == 2:\n",
    "                node.input[0] = node.input[1]\n",
    "                del node.input[1]\n",
    "                \n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('output-tiny-bert-squad/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "tf.set_random_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "pb = 'output-tiny-bert-squad/frozen_model.pb'\n",
    "\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "    \n",
    "inputs = ['Placeholder', 'Placeholder_1', 'Placeholder_2', 'Placeholder_3']\n",
    "outputs = ['start_top_log_probs',\n",
    " 'start_top_index',\n",
    " 'end_top_log_probs',\n",
    " 'end_top_index',\n",
    " 'cls_logits',\n",
    " 'logits_vectorize']\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                           inputs,\n",
    "                                           outputs, transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('output-tiny-bert-squad/frozen_model.pb.quantized')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_nodes = {i: g.get_tensor_by_name(f'import/{i}:0') for i in inputs}\n",
    "output_nodes = {i: g.get_tensor_by_name(f'import/{i}:0') for i in outputs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'import/start_top_log_probs:0' shape=(?, 5) dtype=float32>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_nodes['start_top_log_probs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = [batch_ids, batch_segment, batch_masks, p_mask]\n",
    "b = {input_nodes[i]: b[no] for no, i in enumerate(inputs)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "o = test_sess.run(\n",
    "    output_nodes, feed_dict = b,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'start_top_log_probs': array([[-0.06966875, -3.240222  , -4.473772  , -5.298744  , -5.6682377 ],\n",
       "        [-0.25609264, -2.318633  , -2.9327104 , -3.522881  , -4.5987473 ]],\n",
       "       dtype=float32), 'start_top_index': array([[56, 54, 22, 55,  0],\n",
       "        [39, 38, 46,  0, 44]], dtype=int32), 'end_top_log_probs': array([[-6.5866276e-03, -5.1756473e+00, -7.5562983e+00, -9.1460800e+00,\n",
       "         -9.4442930e+00, -4.5594490e-01, -1.7612027e+00, -2.4332027e+00,\n",
       "         -2.4676924e+00, -4.4934702e+00, -1.8880528e-01, -2.3249977e+00,\n",
       "         -3.7639778e+00, -3.7674901e+00, -4.8629518e+00, -3.7221590e-01,\n",
       "         -1.5458037e+00, -2.7700737e+00, -3.8536804e+00, -5.2050881e+00,\n",
       "         -3.6656349e-03, -6.7355733e+00, -8.3460884e+00, -9.1573057e+00,\n",
       "         -9.2870998e+00],\n",
       "        [-1.8938763e-01, -1.7753700e+00, -6.5651364e+00, -7.5663762e+00,\n",
       "         -8.2851267e+00, -1.1159354e-01, -2.4293358e+00, -5.1858535e+00,\n",
       "         -5.7707458e+00, -6.0979123e+00, -1.4081408e-02, -4.3420844e+00,\n",
       "         -8.3630133e+00, -8.6160736e+00, -8.9481201e+00, -5.0400908e-04,\n",
       "         -9.1286707e+00, -1.0894562e+01, -1.1018955e+01, -1.1310320e+01,\n",
       "         -9.2015363e-02, -2.6028233e+00, -5.3075905e+00, -6.2781239e+00,\n",
       "         -6.7195759e+00]], dtype=float32), 'end_top_index': array([[56, 57, 55, 54, 45, 56, 57, 55, 54, 53, 22, 25, 30, 27, 45, 56,\n",
       "         57, 55, 54, 51,  0, 56, 57, 89, 45],\n",
       "        [46, 42, 43, 58, 57, 46, 42, 43, 58, 57, 46, 42, 58, 43, 57,  0,\n",
       "         46, 58, 42, 57, 46, 42, 43, 58, 52]], dtype=int32), 'cls_logits': array([-3.780888 , -3.3126533], dtype=float32), 'logits_vectorize': array([[[ 0.9446959 ,  0.5307126 ,  0.13359801, ..., -1.4401951 ,\n",
       "          -0.23649111, -2.4114678 ],\n",
       "         [-0.7375185 , -1.2953272 , -0.7870926 , ..., -0.71655416,\n",
       "           1.8732646 ,  1.5296088 ],\n",
       "         [-0.52166903, -0.02243264, -0.4417743 , ..., -0.8854591 ,\n",
       "           0.4548171 ,  1.6796644 ],\n",
       "         ...,\n",
       "         [-0.90625185, -0.7837918 , -1.596335  , ..., -0.8096317 ,\n",
       "           2.9028308 , -1.5527605 ],\n",
       "         [-1.2742921 , -0.71831214, -1.7787229 , ..., -0.7018119 ,\n",
       "           3.0574183 , -1.6671433 ],\n",
       "         [-0.9762547 , -0.5941592 , -1.5555592 , ..., -0.80073285,\n",
       "           3.4092994 , -0.8913306 ]],\n",
       " \n",
       "        [[ 0.8195353 ,  0.8461061 ,  0.63004863, ..., -1.5194983 ,\n",
       "          -0.13577849, -1.5761855 ],\n",
       "         [-0.26177305, -0.47948962,  0.46825445, ..., -1.2120806 ,\n",
       "          -2.8050613 ,  0.73509526],\n",
       "         [-1.0232806 , -0.1463594 ,  0.52070445, ..., -1.6513393 ,\n",
       "          -1.9353168 ,  1.6403039 ],\n",
       "         ...,\n",
       "         [-0.73485994, -0.97864646, -1.5050339 , ..., -0.61353385,\n",
       "           2.3036792 , -1.3338897 ],\n",
       "         [-1.1920509 , -0.87304324, -1.5777805 , ..., -0.52295774,\n",
       "           2.4495692 , -1.2070919 ],\n",
       "         [-0.9147154 , -0.81580937, -1.326014  , ..., -0.65755427,\n",
       "           2.7133627 , -0.9742095 ]]], dtype=float32)}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
